Antenna Season Report Notebook

Josh Dillon, Last Revised January 2022

This notebook examines an individual antenna's performance over a whole season. This notebook parses information from each nightly rtp_summarynotebook (as saved to .csvs) and builds a table describing antenna performance. It also reproduces per-antenna plots from each auto_metrics notebook pertinent to the specific antenna.

import os
from IPython.display import display, HTML
display(HTML("<style>.container { width:100% !important; }</style>"))

# If you want to run this notebook locally, copy the output of the next cell into the next line of this cell.
# antenna = "54"
# csv_folder = '/lustre/aoc/projects/hera/H5C/H5C_Notebooks/_rtp_summary_'
# auto_metrics_folder = '/lustre/aoc/projects/hera/H5C/H5C_Notebooks/auto_metrics_inspect'
# os.environ["ANTENNA"] = antenna
# os.environ["CSV_FOLDER"] = csv_folder
# os.environ["AUTO_METRICS_FOLDER"] = auto_metrics_folder

# Use environment variables to figure out path to the csvs and auto_metrics
antenna = os.environ["ANTENNA"]
csv_folder = os.environ["CSV_FOLDER"]
auto_metrics_folder = os.environ["AUTO_METRICS_FOLDER"]
print(f'antenna = "{antenna}"')
print(f'csv_folder = "{csv_folder}"')
print(f'auto_metrics_folder = "{auto_metrics_folder}"')

antenna = "118"
csv_folder = "/home/obs/src/H5C_Notebooks/_rtp_summary_"
auto_metrics_folder = "/home/obs/src/H5C_Notebooks/auto_metrics_inspect"

display(HTML(f'<h1 style=font-size:50px><u>Antenna {antenna} Report</u><p></p></h1>'))

Antenna 118 Report

import numpy as np
import pandas as pd
pd.set_option('display.max_rows', 1000)
import glob
import re
from hera_notebook_templates.utils import status_colors, Antenna

# load csvs and auto_metrics htmls in reverse chronological order
csvs = sorted(glob.glob(os.path.join(csv_folder, '*.csv')))[::-1]
print(f'Found {len(csvs)} csvs in {csv_folder}')
auto_metric_htmls = sorted(glob.glob(auto_metrics_folder + '/auto_metrics_inspect_*.html'))[::-1]
print(f'Found {len(auto_metric_htmls)} auto_metrics notebooks in {auto_metrics_folder}')

Found 237 csvs in /home/obs/src/H5C_Notebooks/_rtp_summary_
Found 224 auto_metrics notebooks in /home/obs/src/H5C_Notebooks/auto_metrics_inspect

# Per-season options
mean_round_modz_cut = 4
dead_cut = 0.4
crossed_cut = 0.0

def jd_to_summary_url(jd):
    return f'https://htmlpreview.github.io/?https://github.com/HERA-Team/H5C_Notebooks/blob/main/_rtp_summary_/rtp_summary_{jd}.html'

def jd_to_auto_metrics_url(jd):
    return f'https://htmlpreview.github.io/?https://github.com/HERA-Team/H5C_Notebooks/blob/main/auto_metrics_inspect/auto_metrics_inspect_{jd}.html'

Load relevant info from summary CSVs

this_antenna = None
jds = []

# parse information about antennas and nodes
for csv in csvs:
    df = pd.read_csv(csv)
    for n in range(len(df)):
        # Add this day to the antenna
        row = df.loc[n]
        antnum = row['Ant']
        if antnum != int(antenna):
            continue
            
        if this_antenna is None:
            this_antenna = Antenna(row['Ant'], row['Node'])
        jd = [int(s) for s in re.split('_|\.', csv) if s.isdigit()][-1]
        jds.append(jd)
        this_antenna.add_day(jd, row)
        break

---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
~/anaconda/envs/RTP/lib/python3.7/site-packages/pandas/core/indexes/base.py in get_loc(self, key, method, tolerance)
   3360             try:
-> 3361                 return self._engine.get_loc(casted_key)
   3362             except KeyError as err:

~/anaconda/envs/RTP/lib/python3.7/site-packages/pandas/_libs/index.pyx in pandas._libs.index.IndexEngine.get_loc()

~/anaconda/envs/RTP/lib/python3.7/site-packages/pandas/_libs/index.pyx in pandas._libs.index.IndexEngine.get_loc()

pandas/_libs/hashtable_class_helper.pxi in pandas._libs.hashtable.PyObjectHashTable.get_item()

pandas/_libs/hashtable_class_helper.pxi in pandas._libs.hashtable.PyObjectHashTable.get_item()

KeyError: 'Ant'

The above exception was the direct cause of the following exception:

KeyError                                  Traceback (most recent call last)
/tmp/ipykernel_3302/3863632283.py in <module>
      8         # Add this day to the antenna
      9         row = df.loc[n]
---> 10         antnum = row['Ant']
     11         if antnum != int(antenna):
     12             continue

~/anaconda/envs/RTP/lib/python3.7/site-packages/pandas/core/series.py in __getitem__(self, key)
    940 
    941         elif key_is_scalar:
--> 942             return self._get_value(key)
    943 
    944         if is_hashable(key):

~/anaconda/envs/RTP/lib/python3.7/site-packages/pandas/core/series.py in _get_value(self, label, takeable)
   1049 
   1050         # Similar to Index.get_value, but we do not fall back to positional
-> 1051         loc = self.index.get_loc(label)
   1052         return self.index._get_values_for_loc(self, loc, label)
   1053 

~/anaconda/envs/RTP/lib/python3.7/site-packages/pandas/core/indexes/base.py in get_loc(self, key, method, tolerance)
   3361                 return self._engine.get_loc(casted_key)
   3362             except KeyError as err:
-> 3363                 raise KeyError(key) from err
   3364 
   3365         if is_scalar(key) and isna(key) and not self.hasnans:

KeyError: 'Ant'

# build dataframe
to_show = {'JDs': [f'<a href="{jd_to_summary_url(jd)}" target="_blank">{jd}</a>' for jd in jds]}
to_show['A Priori Status'] = [this_antenna.statuses[jd] for jd in jds]

df = pd.DataFrame(to_show)

# create bar chart columns for flagging percentages:
bar_cols = {}
bar_cols['Auto Metrics Flags'] = [this_antenna.auto_flags[jd] for jd in jds]
bar_cols[f'Dead Fraction in Ant Metrics (Jee)'] = [this_antenna.dead_flags_Jee[jd] for jd in jds]
bar_cols[f'Dead Fraction in Ant Metrics (Jnn)'] = [this_antenna.dead_flags_Jnn[jd] for jd in jds]
bar_cols['Crossed Fraction in Ant Metrics'] = [this_antenna.crossed_flags[jd] for jd in jds]
bar_cols['Flag Fraction Before Redcal'] = [this_antenna.flags_before_redcal[jd] for jd in jds]
bar_cols['Flagged By Redcal chi^2 Fraction'] = [this_antenna.redcal_flags[jd] for jd in jds]
for col in bar_cols:
    df[col] = bar_cols[col]

z_score_cols = {}
z_score_cols['ee Shape Modified Z-Score'] = [this_antenna.ee_shape_zs[jd] for jd in jds]
z_score_cols['nn Shape Modified Z-Score'] = [this_antenna.nn_shape_zs[jd] for jd in jds]
z_score_cols['ee Power Modified Z-Score'] = [this_antenna.ee_power_zs[jd] for jd in jds]
z_score_cols['nn Power Modified Z-Score'] = [this_antenna.nn_power_zs[jd] for jd in jds]
z_score_cols['ee Temporal Variability Modified Z-Score'] = [this_antenna.ee_temp_var_zs[jd] for jd in jds]
z_score_cols['nn Temporal Variability Modified Z-Score'] = [this_antenna.nn_temp_var_zs[jd] for jd in jds]
z_score_cols['ee Temporal Discontinuties Modified Z-Score'] = [this_antenna.ee_temp_discon_zs[jd] for jd in jds]
z_score_cols['nn Temporal Discontinuties Modified Z-Score'] = [this_antenna.nn_temp_discon_zs[jd] for jd in jds]
for col in z_score_cols:
    df[col] = z_score_cols[col]

ant_metrics_cols = {}
ant_metrics_cols['Average Dead Ant Metric (Jee)'] = [this_antenna.Jee_dead_metrics[jd] for jd in jds]
ant_metrics_cols['Average Dead Ant Metric (Jnn)'] = [this_antenna.Jnn_dead_metrics[jd] for jd in jds]
ant_metrics_cols['Average Crossed Ant Metric'] = [this_antenna.crossed_metrics[jd] for jd in jds]
for col in ant_metrics_cols:
    df[col] = ant_metrics_cols[col]

redcal_cols = {}
redcal_cols['Median chi^2 Per Antenna (Jee)'] = [this_antenna.Jee_chisqs[jd] for jd in jds]
redcal_cols['Median chi^2 Per Antenna (Jnn)'] = [this_antenna.Jnn_chisqs[jd] for jd in jds]   
for col in redcal_cols:
    df[col] = redcal_cols[col]

# style dataframe
table = df.style.hide_index()\
          .applymap(lambda val: f'background-color: {status_colors[val]}' if val in status_colors else '', subset=['A Priori Status']) \
          .background_gradient(cmap='viridis', vmax=mean_round_modz_cut * 3, vmin=0, axis=None, subset=list(z_score_cols.keys())) \
          .background_gradient(cmap='bwr_r', vmin=dead_cut-.25, vmax=dead_cut+.25, axis=0, subset=list([col for col in ant_metrics_cols if 'dead' in col.lower()])) \
          .background_gradient(cmap='bwr_r', vmin=crossed_cut-.25, vmax=crossed_cut+.25, axis=0, subset=list([col for col in ant_metrics_cols if 'crossed' in col.lower()])) \
          .background_gradient(cmap='plasma', vmax=4, vmin=1, axis=None, subset=list(redcal_cols.keys())) \
          .applymap(lambda val: 'font-weight: bold' if val < dead_cut else '', subset=list([col for col in ant_metrics_cols if 'dead' in col.lower()])) \
          .applymap(lambda val: 'font-weight: bold' if val < crossed_cut else '', subset=list([col for col in ant_metrics_cols if 'crossed' in col.lower()])) \
          .applymap(lambda val: 'font-weight: bold' if val > mean_round_modz_cut else '', subset=list(z_score_cols.keys())) \
          .applymap(lambda val: 'color: red' if val > mean_round_modz_cut else '', subset=list(z_score_cols.keys())) \
          .bar(subset=list(bar_cols.keys()), vmin=0, vmax=1) \
          .format({col: '{:,.4f}'.format for col in z_score_cols}) \
          .format({col: '{:,.4f}'.format for col in ant_metrics_cols}) \
          .format('{:,.2%}', na_rep='-', subset=list(bar_cols.keys())) \
          .set_table_styles([dict(selector="th",props=[('max-width', f'70pt')])]) 

Table 1: Per-Night RTP Summary Info For This Atenna

This table reproduces each night's row for this antenna from the RTP Summary notebooks. For more info on the columns, see those notebooks, linked in the JD column.

display(HTML(f'<h2>Antenna {antenna}, Node {this_antenna.node}:</h2>'))
HTML(table.render(render_links=True, escape=False))

---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
/tmp/ipykernel_3302/880101968.py in <module>
----> 1 display(HTML(f'<h2>Antenna {antenna}, Node {this_antenna.node}:</h2>'))
      2 HTML(table.render(render_links=True, escape=False))

AttributeError: 'NoneType' object has no attribute 'node'

Load antenna metric spectra and waterfalls from auto_metrics notebooks.

htmls_to_display = []
for am_html in auto_metric_htmls:
    html_to_display = ''
    # read html into a list of lines
    with open(am_html) as f:
        lines = f.readlines()
    
    # find section with this antenna's metric plots and add to html_to_display
    jd = [int(s) for s in re.split('_|\.', am_html) if s.isdigit()][-1]
    try:
        section_start_line = lines.index(f'<h2>Antenna {antenna}: {jd}</h2>\n')
    except ValueError:
        continue
    html_to_display += lines[section_start_line].replace(str(jd), f'<a href="{jd_to_auto_metrics_url(jd)}" target="_blank">{jd}</a>')
    for line in lines[section_start_line + 1:]:
        html_to_display += line
        if '<hr' in line:
            htmls_to_display.append(html_to_display)
            break

Figure 1: Antenna autocorrelation metric spectra and waterfalls.

These figures are reproduced from auto_metrics notebooks. For more info on the specific plots and metrics, see those notebooks (linked at the JD). The most recent 100 days (at most) are shown.

for i, html_to_display in enumerate(htmls_to_display):
    if i == 100:
        break
    display(HTML(html_to_display))

Antenna 118: 2459787

Ant	Node	A Priori Status	Worst Metric	Worst Modified Z-Score	ee Shape Modified Z-Score	nn Shape Modified Z-Score	ee Power Modified Z-Score	nn Power Modified Z-Score	ee Temporal Variability Modified Z-Score	nn Temporal Variability Modified Z-Score	ee Temporal Discontinuties Modified Z-Score	nn Temporal Discontinuties Modified Z-Score
118	N07	digital_ok	ee Shape	2.727325	2.727325	1.437319	1.866101	0.427121	1.500725	0.244917	2.153838	0.785188

---

Antenna 118: 2459786

Ant	Node	A Priori Status	Worst Metric	Worst Modified Z-Score	nn Shape Modified Z-Score	ee Shape Modified Z-Score	nn Power Modified Z-Score	ee Power Modified Z-Score	nn Temporal Variability Modified Z-Score	ee Temporal Variability Modified Z-Score	nn Temporal Discontinuties Modified Z-Score	ee Temporal Discontinuties Modified Z-Score
118	N07	digital_ok	nn Temporal Discontinuties	7.181871	1.908054	4.058978	0.665554	2.876671	0.130584	3.723308	7.181871	6.011977

---

Antenna 118: 2459785

Ant	Node	A Priori Status	Worst Metric	Worst Modified Z-Score	nn Shape Modified Z-Score	ee Shape Modified Z-Score	nn Power Modified Z-Score	ee Power Modified Z-Score	nn Temporal Variability Modified Z-Score	ee Temporal Variability Modified Z-Score	nn Temporal Discontinuties Modified Z-Score	ee Temporal Discontinuties Modified Z-Score
118	N07	digital_ok	ee Power	8.837049	1.517891	2.386754	-0.462039	8.837049	0.619276	3.879273	1.164861	0.548789

---

Antenna 118: 2459784

Ant	Node	A Priori Status	Worst Metric	Worst Modified Z-Score	ee Shape Modified Z-Score	nn Shape Modified Z-Score	ee Power Modified Z-Score	nn Power Modified Z-Score	ee Temporal Variability Modified Z-Score	nn Temporal Variability Modified Z-Score	ee Temporal Discontinuties Modified Z-Score	nn Temporal Discontinuties Modified Z-Score
118	N07	digital_ok	ee Temporal Variability	7.871495	3.001645	1.873509	6.862795	0.232989	7.871495	0.166104	1.091775	3.859584

---

Antenna 118: 2459783

Ant	Node	A Priori Status	Worst Metric	Worst Modified Z-Score	nn Shape Modified Z-Score	ee Shape Modified Z-Score	nn Power Modified Z-Score	ee Power Modified Z-Score	nn Temporal Variability Modified Z-Score	ee Temporal Variability Modified Z-Score	nn Temporal Discontinuties Modified Z-Score	ee Temporal Discontinuties Modified Z-Score
118	N07	digital_ok	ee Shape	2.512890	1.174603	2.512890	-1.418553	1.163121	-1.195223	2.061870	1.308646	0.290130

---

Antenna 118: 2459782

Ant	Node	A Priori Status	Worst Metric	Worst Modified Z-Score	nn Shape Modified Z-Score	ee Shape Modified Z-Score	nn Power Modified Z-Score	ee Power Modified Z-Score	nn Temporal Variability Modified Z-Score	ee Temporal Variability Modified Z-Score	nn Temporal Discontinuties Modified Z-Score	ee Temporal Discontinuties Modified Z-Score
118	N07	digital_ok	ee Shape	2.761449	1.342720	2.761449	-0.808168	0.594867	-1.014983	0.706025	-0.227069	-0.417420

---

Antenna 118: 2459781

Ant	Node	A Priori Status	Worst Metric	Worst Modified Z-Score	ee Shape Modified Z-Score	nn Shape Modified Z-Score	ee Power Modified Z-Score	nn Power Modified Z-Score	ee Temporal Variability Modified Z-Score	nn Temporal Variability Modified Z-Score	ee Temporal Discontinuties Modified Z-Score	nn Temporal Discontinuties Modified Z-Score
118	N07	digital_ok	nn Temporal Variability	15.536031	-0.575675	0.446567	-0.461975	0.181251	15.292783	15.536031	0.068251	0.228706

---

Antenna 118: 2459778

Ant	Node	A Priori Status	Worst Metric	Worst Modified Z-Score	nn Shape Modified Z-Score	ee Shape Modified Z-Score	nn Power Modified Z-Score	ee Power Modified Z-Score	nn Temporal Variability Modified Z-Score	ee Temporal Variability Modified Z-Score	nn Temporal Discontinuties Modified Z-Score	ee Temporal Discontinuties Modified Z-Score
118	N07	digital_ok	ee Shape	1.491992	0.347015	1.491992	-1.000884	0.730303	-0.868178	-0.182969	0.688903	0.135413

---

Antenna 118: 2459776

Ant	Node	A Priori Status	Worst Metric	Worst Modified Z-Score	ee Shape Modified Z-Score	nn Shape Modified Z-Score	ee Power Modified Z-Score	nn Power Modified Z-Score	ee Temporal Variability Modified Z-Score	nn Temporal Variability Modified Z-Score	ee Temporal Discontinuties Modified Z-Score	nn Temporal Discontinuties Modified Z-Score
118	N07	digital_ok	ee Temporal Discontinuties	1.947524	1.942734	1.488058	0.421715	-0.704494	-0.657607	-0.695761	1.947524	1.786886

---

Antenna 118: 2459773

Ant	Node	A Priori Status	Worst Metric	Worst Modified Z-Score	nn Shape Modified Z-Score	ee Shape Modified Z-Score	nn Power Modified Z-Score	ee Power Modified Z-Score	nn Temporal Variability Modified Z-Score	ee Temporal Variability Modified Z-Score	nn Temporal Discontinuties Modified Z-Score	ee Temporal Discontinuties Modified Z-Score
118	N07	digital_ok	ee Temporal Variability	1.844325	1.037270	1.024432	-0.323187	1.754474	-0.684379	1.844325	0.954175	-0.575655

---

Antenna 118: 2459772

Ant	Node	A Priori Status	Worst Metric	Worst Modified Z-Score	ee Shape Modified Z-Score	nn Shape Modified Z-Score	ee Power Modified Z-Score	nn Power Modified Z-Score	ee Temporal Variability Modified Z-Score	nn Temporal Variability Modified Z-Score	ee Temporal Discontinuties Modified Z-Score	nn Temporal Discontinuties Modified Z-Score
118	N07	digital_ok	ee Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan

---

Antenna 118: 2459771

Ant	Node	A Priori Status	Worst Metric	Worst Modified Z-Score	nn Shape Modified Z-Score	ee Shape Modified Z-Score	nn Power Modified Z-Score	ee Power Modified Z-Score	nn Temporal Variability Modified Z-Score	ee Temporal Variability Modified Z-Score	nn Temporal Discontinuties Modified Z-Score	ee Temporal Discontinuties Modified Z-Score
118	N07	digital_ok	nn Temporal Discontinuties	3.175964	0.252336	-0.259660	0.029843	-1.164331	-0.064553	-1.171780	3.175964	0.730347

---

Antenna 118: 2459770

Ant	Node	A Priori Status	Worst Metric	Worst Modified Z-Score	nn Shape Modified Z-Score	ee Shape Modified Z-Score	nn Power Modified Z-Score	ee Power Modified Z-Score	nn Temporal Variability Modified Z-Score	ee Temporal Variability Modified Z-Score	nn Temporal Discontinuties Modified Z-Score	ee Temporal Discontinuties Modified Z-Score
118	N07	digital_ok	nn Temporal Discontinuties	7.138536	1.081766	1.043616	0.145474	1.725896	-0.229381	1.480561	7.138536	0.149500

---

Antenna 118: 2459769

Ant	Node	A Priori Status	Worst Metric	Worst Modified Z-Score	nn Shape Modified Z-Score	ee Shape Modified Z-Score	nn Power Modified Z-Score	ee Power Modified Z-Score	nn Temporal Variability Modified Z-Score	ee Temporal Variability Modified Z-Score	nn Temporal Discontinuties Modified Z-Score	ee Temporal Discontinuties Modified Z-Score
118	N07	digital_ok	ee Temporal Variability	2.741047	1.133086	1.474971	0.016974	2.528788	-0.420812	2.741047	1.204813	-0.217731

---

Antenna 118: 2459768

Ant	Node	A Priori Status	Worst Metric	Worst Modified Z-Score	ee Shape Modified Z-Score	nn Shape Modified Z-Score	ee Power Modified Z-Score	nn Power Modified Z-Score	ee Temporal Variability Modified Z-Score	nn Temporal Variability Modified Z-Score	ee Temporal Discontinuties Modified Z-Score	nn Temporal Discontinuties Modified Z-Score
118	N07	digital_ok	ee Power	11.837305	7.059928	5.531814	11.837305	9.632527	11.811262	7.780067	-4.035068	-4.037021

---

Antenna 118: 2459767

Ant	Node	A Priori Status	Worst Metric	Worst Modified Z-Score	ee Shape Modified Z-Score	nn Shape Modified Z-Score	ee Power Modified Z-Score	nn Power Modified Z-Score	ee Temporal Variability Modified Z-Score	nn Temporal Variability Modified Z-Score	ee Temporal Discontinuties Modified Z-Score	nn Temporal Discontinuties Modified Z-Score
118	N07	digital_ok	ee Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan

---

Antenna 118: 2459766

Ant	Node	A Priori Status	Worst Metric	Worst Modified Z-Score	nn Shape Modified Z-Score	ee Shape Modified Z-Score	nn Power Modified Z-Score	ee Power Modified Z-Score	nn Temporal Variability Modified Z-Score	ee Temporal Variability Modified Z-Score	nn Temporal Discontinuties Modified Z-Score	ee Temporal Discontinuties Modified Z-Score
118	N07	digital_ok	nn Temporal Discontinuties	1.492886	0.597369	-0.218601	0.256356	-1.258498	0.153239	-1.534574	1.492886	0.329838

---

Antenna 118: 2459765

Ant	Node	A Priori Status	Worst Metric	Worst Modified Z-Score	nn Shape Modified Z-Score	ee Shape Modified Z-Score	nn Power Modified Z-Score	ee Power Modified Z-Score	nn Temporal Variability Modified Z-Score	ee Temporal Variability Modified Z-Score	nn Temporal Discontinuties Modified Z-Score	ee Temporal Discontinuties Modified Z-Score
118	N07	digital_ok	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan

---

Antenna 118: 2459763

Ant	Node	A Priori Status	Worst Metric	Worst Modified Z-Score	ee Shape Modified Z-Score	nn Shape Modified Z-Score	ee Power Modified Z-Score	nn Power Modified Z-Score	ee Temporal Variability Modified Z-Score	nn Temporal Variability Modified Z-Score	ee Temporal Discontinuties Modified Z-Score	nn Temporal Discontinuties Modified Z-Score
118	N07	digital_ok	nn Shape	1.299378	0.958415	1.299378	0.547062	-0.020068	0.648598	0.778877	0.037809	0.062045

---